{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Import Modules",
   "id": "8a1d7e2051fa176d"
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Start with imports - ask ChatGPT to explain any package that you don't know\n",
    "\n",
    "import os\n",
    "from pathlib import Path\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "from openai import OpenAI\n",
    "from anthropic import Anthropic\n",
    "from pydantic_settings import BaseSettings\n",
    "from pydantic import Field"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Load Environment Variables",
   "id": "4891fb607ab40782"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "load_dotenv()",
   "id": "1e488414d791227a",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Set Run Variables",
   "id": "f3f2f6db181bd69"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# load the note text\n",
    "note_text = Path(\"note_text.txt\").read_text(encoding=\"utf-8\")\n",
    "\n",
    "# the field name will look for an environment variable with that name\n",
    "# e.g. openai_api_key will look for OPENAI_API_KEY\n",
    "class ApiSettings(BaseSettings):\n",
    "    openai_api_key: str | None = Field(None)\n",
    "    anthropic_api_key: str | None = Field(None)\n",
    "    google_api_key: str | None = Field(None)\n",
    "    deepseek_api_key: str | None = Field(None)\n",
    "    groq_api_key: str | None = Field(None)\n",
    "\n",
    "api_settings = ApiSettings()"
   ],
   "id": "e7f4bd6f62b4911a",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Add Necessary Functions",
   "id": "50427466b78acc4"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "def get_model_response(model_name: str, question: str, **kwargs):\n",
    "\n",
    "    messages = [{\"role\": \"user\", \"content\": question}]\n",
    "\n",
    "    match model_name:\n",
    "        case name if name.startswith(\"gpt\"):\n",
    "            print(f\"Running OpenAI model {model_name}...\")\n",
    "            openai = OpenAI()\n",
    "            response = openai.chat.completions.create(\n",
    "                model=model_name,\n",
    "                messages=messages,\n",
    "            )\n",
    "            return response.choices[0].message.content\n",
    "\n",
    "        case name if name.startswith(\"claude\"):\n",
    "            print(f\"Running Anthropic model {model_name}...\")\n",
    "            anthropic = Anthropic()\n",
    "            response = anthropic.messages.create(\n",
    "                model=model_name,\n",
    "                messages=messages,\n",
    "                max_tokens=kwargs.get(\"max_tokens\", 1000)\n",
    "            )\n",
    "            return response.content[0].text\n",
    "\n",
    "        case _:\n",
    "\n",
    "\n",
    "            return \"Model not supported.\""
   ],
   "id": "892bc8018727df2b",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Check API Keys",
   "id": "29b2f181e66e1149"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Check for API Keys\n",
    "for key_name, key_value in api_settings.model_dump().items():\n",
    "    if key_value:\n",
    "        print(f\"{key_name} exists and begins {key_value[:8]}\")\n",
    "    else:\n",
    "        print(f\"{key_name} not set\")"
   ],
   "id": "1750d9e13b0cf8b8",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Setup Initial Question",
   "id": "3372c9600e6dc2d"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "diag_question = \"What diagnoses are mentioned in the following medical note?\\n\\n\"\n",
    "diag_question += \"Respond with a JSON array of diagnosis strings with the corresponding ICD10CM code. Do not include any other text. Please don't respond in markdown\\n\\n\"\n",
    "diag_question += f\"Medical Note:\\n{note_text}\""
   ],
   "id": "5f3134193b5379c9",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Evaluate Several Models",
   "id": "f31d57289dc17247"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "models = [\n",
    "    \"gpt-5-mini\",\n",
    "    \"gpt-5-nano\",\n",
    "    \"claude-sonnet-4-5\",\n",
    "    \"claude-haiku-4-5\"\n",
    "]\n",
    "\n",
    "answers = []\n",
    "\n",
    "for model in models:\n",
    "    response = get_model_response(model, question=diag_question, max_tokens=1000)\n",
    "    answers.append({\"model_name\": model, \"response\": response})\n"
   ],
   "id": "80bdf93c4fb92fab",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Combine Responses",
   "id": "f0901e23015cc233"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "together = \"\"\n",
    "for index, answer in enumerate(answers, start=1):\n",
    "    together += f\"# Response from competitor {index}\\n\\n\"\n",
    "    together += answer.get(\"response\") + \"\\n\\n\""
   ],
   "id": "ae90114944620df0",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Evaluate Responses",
   "id": "2a08b4e04f29f188"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "judge = f\"\"\"You are judging a competition between {len(models)} competitors.\n",
    "Each model has been given this question:\n",
    "\n",
    "{diag_question}\n",
    "\n",
    "Your job is to evaluate each response for clarity and strength of argument, and rank them in order of best to worst.\n",
    "Respond with JSON, and only JSON, with the following format:\n",
    "{{\"results\": [\"best competitor number\", \"second best competitor number\", \"third best competitor number\", ...]}}\n",
    "\n",
    "Here are the responses from each competitor:\n",
    "\n",
    "{together}\n",
    "\n",
    "Now respond with the JSON with the ranked order of the competitors, nothing else. Do not include markdown formatting or code blocks.\"\"\""
   ],
   "id": "494dda644c4e9cfc",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Run Evaluation",
   "id": "65143deaf12abb0f"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "eval_results = get_model_response(model_name=\"gpt-5-nano\", question=judge, max_tokens=1000)\n",
    "print(eval_results)"
   ],
   "id": "55591985890f97d7",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Show Leaderboard",
   "id": "1e8510a6a9b93784"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "results_dict = json.loads(eval_results)\n",
    "ranks = results_dict[\"results\"]\n",
    "for rank_num, rank_result in enumerate(ranks, start=1):\n",
    "    competitor = answers[ranks.index(str(rank_num))]\n",
    "    print(f\"Rank {rank_num}: {competitor.get('model_name')}\")"
   ],
   "id": "6d111e8927f8fe25",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
